from selenium import webdriver
import time
import sys
from selenium.webdriver.chrome.options import Options

# 文件路径
fpath = 'C:\\data\\CNVD_demo2\\webBugs.json'

# 循环获取漏洞列表
from lxml import etree
def parse_list(base_url,driver):
    page_url = 'https://www.cnvd.org.cn/flaw/typeResult?typeId=29&max=20&offset=120'
    start_url = 'https://www.cnvd.org.cn/flaw/show/CNVD-2020-03249'
    fp = open(fpath, 'a')
    driver.get(page_url)
    print('爬取列表页...', driver.current_url)
    time.sleep(2)
    html = etree.HTML(driver.page_source)
    detail_urls = html.xpath("//a[contains(@href,'CNVD')]/@href")
    flag = 0
    for url in detail_urls:
        full_url = base_url + url
        if full_url == start_url and flag == 0:
            flag = 1
            parse_detail(full_url, base_url, fp, driver)
            time.sleep(2)
            continue
        if flag==1:
            parse_detail(full_url, base_url, fp, driver)
            time.sleep(2)
    fp.close()


# 爬取漏洞详情
def parse_detail(url,base_url,fp, driver):
    driver.get(url)
    html = driver.page_source
    print("爬取详情页...", driver.current_url)
    time.sleep(3)
    response = etree.HTML(html)

    # 漏洞名称
    title = response.xpath("//div[@class='blkContainerSblk']/h1/text()")[0].strip()

    # 公开日期 str
    date = response.xpath("//table[@class='gg_detail']/tbody//tr[2]/td[2]/text()")[0].strip()

    # CNVD_ID str
    CN_ID = response.xpath("//table[@class='gg_detail']//tr[1]//td[2]/text()")[0].strip()

    # 产品 list
    projects_demo = response.xpath("//table[@class='gg_detail']/tbody//tr[4]/td[2]/text()")
    projects = []
    for demo in projects_demo:
        demo = demo.strip()
        if demo != '':
            projects.append(demo)

    # 描述 str
    contents = response.xpath(
        "//table[@class='gg_detail']//td[contains(text(), '漏洞描述')]/following-sibling::td[1]//text()")
    contents = ''.join([j.strip() for j in contents])

    # 等级 str
    level = response.xpath("//table[@class='gg_detail']/tbody//tr[3]/td[2]/text()[2]")
    if level:
        level = level[0].replace('(', '').strip()
    else:
        level = '未评级'

    # 参考链接 url
    ref_url = response.xpath(
        "//table[@class='gg_detail']//td[contains(text(), '参考链接')]/following-sibling::td[1]/a/@href")

    # 解决方案 str
    solve_contents = response.xpath(
        "//table[@class='gg_detail']//td[contains(text(), '漏洞解决方案')]/following-sibling::td[1]//text()")
    solve_contents = ''.join([j.strip() for j in solve_contents])

    # 厂商补丁 str
    buding = response.xpath(
        "//table[@class='gg_detail']//td[contains(text(), '厂商补丁')]/following-sibling::td[1]/a/@href")
    if buding:
        if isinstance(buding[0], str):
            buding = base_url + buding[0]
    dic = {}
    dic['漏洞名称'] = title
    dic['CNVD-ID'] = CN_ID
    dic['公开日期'] = date
    dic['危害级别'] = level
    dic['影响产品'] = projects
    dic['漏洞描述'] = contents
    dic['参考链接'] = ref_url
    dic['漏洞解决方案'] = solve_contents
    dic['厂商补丁'] = buding
    save_item(dic, fp)


# 保存单个漏洞信息
import json
def save_item(dic, fp):
    str_item = json.dumps(dic, ensure_ascii=False)
    fp.write(str_item+"\n")

# main函数
def main(driver):
    base_url = 'https://www.cnvd.org.cn'
    parse_list(base_url,driver)

if __name__ == '__main__':
    # 配置chrome信息
    chrome_option = Options()
    chrome_option.add_argument("--headless")
    driver = webdriver.Chrome(options=chrome_option)
    main(driver)
    driver.close()